import sys, gzip, json, math, nltk, MySQLdb, os, enchant
from collections import Counter, defaultdict
from MySQLdb import cursors

if __name__ == "__main__":
	conn = MySQLdb.connect(host="mysql.stalidis.com", user="ict4growth", passwd="stanford", db="ict4growth")
	writer = conn.cursor()
	all_books = os.listdir("TFMATRIX")
	total = len(all_books)
	iterator = 0
	while len(all_books) > 0:
		existing = os.listdir("VECTORS")
		while len(existing) > 0:
			iterator += 1
			TFfile = existing.pop()
			tffile = gzip.open("TFMATRIX/"+TFfile,'r')
			tfidfs = gzip.open("VECTORS/"+TFfile,'r')
			writer.execute("""INSERT INTO Books VALUES (%s,%s,%s)""",(TFfile.split(".")[0], tffile.readline(), tfidfs.readline()))
			tffile.close()
			tfidfs.close()
			os.rename("VECTORS/"+TFfile,"MOVED/"+TFfile)
			print "moved ", iterator, " files out of ", total
			print len(existing), " files in buffer"
			all_books.remove(TFfile)
	writer.executemany("""INSERT INTO Books VALUES (%s,%s,%s)""",data)
	conn.commit()
	conn.close()
	sys.exit()


